{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "input_collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import org.apache.spark.sql.SQLContext\n",
      "import org.apache.spark.sql.types.{StructType, StructField, StringType, IntegerType}\n",
      "import org.apache.spark.sql.Row\n",
      "sqlContext: org.apache.spark.sql.SQLContext = org.apache.spark.sql.SQLContext@6a8c4ec0\n",
      "schema: org.apache.spark.sql.types.StructType = StructType(StructField(id,StringType,true), StructField(title,StringType,true), StructField(body,StringType,true), StructField(tags,StringType,true))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"container-fluid\"><div><div class=\"col-md-12\"><div>\n",
       "      <script data-this=\"{&quot;dataId&quot;:&quot;anona9ed09308239ae28c83d465218909539&quot;,&quot;dataInit&quot;:[{&quot;name&quot;:&quot;id&quot;,&quot;dataType&quot;:&quot;StringType&quot;,&quot;nullable&quot;:true,&quot;metadata&quot;:&quot;{}&quot;},{&quot;name&quot;:&quot;title&quot;,&quot;dataType&quot;:&quot;StringType&quot;,&quot;nullable&quot;:true,&quot;metadata&quot;:&quot;{}&quot;},{&quot;name&quot;:&quot;body&quot;,&quot;dataType&quot;:&quot;StringType&quot;,&quot;nullable&quot;:true,&quot;metadata&quot;:&quot;{}&quot;},{&quot;name&quot;:&quot;tags&quot;,&quot;dataType&quot;:&quot;StringType&quot;,&quot;nullable&quot;:true,&quot;metadata&quot;:&quot;{}&quot;}],&quot;genId&quot;:&quot;1810141539&quot;}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/tableChart'], \n",
       "      function(playground, _magictableChart) {\n",
       "        // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n",
       "        // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n",
       "\n",
       "        playground.call(data,\n",
       "                        this\n",
       "                        ,\n",
       "                        {\n",
       "    \"f\": _magictableChart,\n",
       "    \"o\": {\"headers\":[\"name\",\"dataType\",\"nullable\",\"metadata\"],\"width\":600,\"height\":400}\n",
       "  }\n",
       "  \n",
       "                        \n",
       "                        \n",
       "                      );\n",
       "      }\n",
       "    );/*]]>*/</script>\n",
       "    <div>\n",
       "        <label for=\"input-anon59365d12fcdd6c3de914ad37729bca08\">\n",
       "      Max Points\n",
       "    </label><input id=\"input-anon59365d12fcdd6c3de914ad37729bca08\" type=\"number\" name=\"input-anon59365d12fcdd6c3de914ad37729bca08\" data-bind=\"textInput: value, fireChange: true, valueUpdate: 'input'\">\n",
       "      <script data-selector=\"#input-anon59365d12fcdd6c3de914ad37729bca08\" data-this=\"{&quot;valueId&quot;:&quot;anon59365d12fcdd6c3de914ad37729bca08&quot;,&quot;valueInit&quot;:25}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(\n",
       " ['observable', 'knockout'],\n",
       " function (Observable, ko) {\n",
       "   //console.log(\"-----------\")\n",
       "   //console.dir(this);\n",
       "   //console.dir(valueId);\n",
       "   var obs = Observable.makeObservable(valueId)\n",
       "                       .extend({ rateLimit: { //throttle\n",
       "                                   timeout: 500,\n",
       "                                   method: \"notifyWhenChangesStop\"\n",
       "                                 }\n",
       "                               }\n",
       "                       );\n",
       "   ko.applyBindings({\n",
       "     value: obs\n",
       "   }, this);\n",
       "   obs(valueInit);\n",
       " }\n",
       ")/*]]>*/</script>\n",
       "    </input>\n",
       "        <p data-bind=\"text: value\"><script data-this=\"{&quot;valueId&quot;:&quot;anon362151d185b24e074a55d1be4faa5dd3&quot;}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\n",
       "req(\n",
       "['observable', 'knockout'],\n",
       "function (O, ko) {\n",
       "  ko.applyBindings({\n",
       "      value: O.makeObservable(valueId)\n",
       "    },\n",
       "    this\n",
       "  );\n",
       "});\n",
       "        /*]]>*/</script></p> <span style=\"color:red\"><p data-bind=\"text: value\"><script data-this=\"{&quot;valueId&quot;:&quot;anon8dd9ee0836cae56adac9be0da0b76c30&quot;}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\n",
       "req(\n",
       "['observable', 'knockout'],\n",
       "function (O, ko) {\n",
       "  ko.applyBindings({\n",
       "      value: O.makeObservable(valueId)\n",
       "    },\n",
       "    this\n",
       "  );\n",
       "});\n",
       "        /*]]>*/</script></p></span>\n",
       "        <div>\n",
       "        </div>\n",
       "      </div></div></div></div></div>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import org.apache.spark.sql.SQLContext\n",
    "import org.apache.spark.sql.types.{StructType, StructField, StringType, IntegerType}\n",
    "import org.apache.spark.sql.Row\n",
    "\n",
    "val sqlContext = new SQLContext(sc)\n",
    "\n",
    "val schema = StructType(Array(\n",
    "     StructField(\"id\",StringType,true),\n",
    "     StructField(\"title\",StringType,true),\n",
    "     StructField(\"body\",StringType,true),\n",
    "     StructField(\"tags\",StringType,true)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "input_collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df: org.apache.spark.sql.DataFrame = [id: string, title: string, body: string, tags: string]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"df-canvas\">\n",
       "      <script data-this=\"{&quot;dataId&quot;:&quot;anon5e82b780aae79824d36152e22c60cb92&quot;,&quot;partitionIndexId&quot;:&quot;anon7864cfaacc81341f97db2f5b6785074a&quot;,&quot;numPartitions&quot;:241368,&quot;dfSchema&quot;:{&quot;type&quot;:&quot;struct&quot;,&quot;fields&quot;:[{&quot;name&quot;:&quot;id&quot;,&quot;type&quot;:&quot;string&quot;,&quot;nullable&quot;:true,&quot;metadata&quot;:{}},{&quot;name&quot;:&quot;title&quot;,&quot;type&quot;:&quot;string&quot;,&quot;nullable&quot;:true,&quot;metadata&quot;:{}},{&quot;name&quot;:&quot;body&quot;,&quot;type&quot;:&quot;string&quot;,&quot;nullable&quot;:true,&quot;metadata&quot;:{}},{&quot;name&quot;:&quot;tags&quot;,&quot;type&quot;:&quot;string&quot;,&quot;nullable&quot;:true,&quot;metadata&quot;:{}}]}}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/dataframe','../javascripts/notebook/consoleDir'], \n",
       "      function(dataframe, extension) {\n",
       "        dataframe.call(data, this, extension);\n",
       "      }\n",
       "    );/*]]>*/</script>\n",
       "      <link rel=\"stylesheet\" href=\"/assets/stylesheets/ipython/css/dataframe.css\" type=\"text/css\"/>\n",
       "    </div>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val df = sqlContext.read\n",
    ".format(\"com.databricks.spark.csv\")\n",
    ".option(\"mode\",\"DROPMALFORMED\")\n",
    ".schema(schema)\n",
    ".load(\"/home/felipe/auto-tagger/data/stackoverflow/pieces\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "input_collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count: Long = 6034195\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "6034195"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "// old count was 6034194\n",
    "val count = df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "input_collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labelCardinality: Double = 2.885222303886431\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "2.885222303886431"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val labelCardinality = df\n",
    "  .select(df(\"tags\"))\n",
    "  .map{ case Row(tags) => Option(tags) }\n",
    "  .map{ \n",
    "     case Some(tags) => tags.toString.split(\" \").size.toDouble\n",
    "     case None => 0.0\n",
    "    } \n",
    " .mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "input_collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "distinctLabelCount: Long = 42048\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "42048"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val distinctLabelCount = df\n",
    "  .select(df(\"tags\"))\n",
    "  .map { case Row(tags) => Option(tags) }\n",
    "  .filter( opt => opt.isDefined)\n",
    "  .map( opt => opt.get)\n",
    "  .flatMap( tags => tags.toString.split(\" \") )\n",
    "  .map( tag => (tag,1) )\n",
    "  .reduceByKey( (a:Int,b:Int) => a+b )\n",
    "  .map{ case (label,count) => label }\n",
    "  .count()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "input_collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labelSetCount: Long = 2075362\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "2075362"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val labelSetCount = df\n",
    "  .select(df(\"tags\"))\n",
    "  .map { case Row(tags) => Option(tags) }\n",
    "  .filter( opt => opt.isDefined)\n",
    "  .map( opt => opt.get)\n",
    "  .map( tags => (tags.toString.split(\" \").toSet,1) ) // using set rather than sequence\n",
    "  .reduceByKey( (a:Int,b:Int) => a+b )\n",
    "  .map{ case (label,count) => label }\n",
    "  .count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "input_collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "numberOfLabelSetsThatOccurOnlyOnce: Long = 1155192\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "1155192"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val numberOfLabelSetsThatOccurOnlyOnce = df\n",
    "  .select(df(\"tags\"))\n",
    "  .map { case Row(tags) => Option(tags) }\n",
    "  .filter( opt => opt.isDefined)\n",
    "  .map( opt => opt.get)\n",
    "  .map( tags => (tags.toString.split(\" \").toSeq,1) )\n",
    "  .reduceByKey( (a:Int,b:Int) => a+b )\n",
    "  .filter{ case(label,count) => count == 1 }\n",
    "  .count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "input_collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "labelDensity with and without scientific notation:\n",
      "\n",
      "labelDensity: Double = 0.0000013902 \n",
      "labelDensity: Double = 1.3902260443654798E-6\n"
     ]
    },
    {
     "data": {
      "text/html": []
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "println(\"labelDensity with and without scientific notation:\\n\")\n",
    "val labelDensity = labelCardinality / labelSetCount\n",
    "print(f\"labelDensity: Double = $labelDensity%.10f \\n\")"
   ]
  }
 ],
 "metadata": {
  "auto_save_timestamp": "1969-12-31T21:00:00.000Z",
  "customArgs": null,
  "customDeps": null,
  "customImports": null,
  "customLocalRepo": null,
  "customRepos": null,
  "customSparkConf": null,
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "name": "characterizing-so-dataset",
  "trusted": true,
  "user_save_timestamp": "1969-12-31T21:00:00.000Z"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
